# -*- coding: utf-8 -*-
import scrapy
from scrapy import Selector
import re

from moose_poetry_spider.items import MoosePoetryRankingItem

'''
诗词排名
https://www.shicimingju.com/paiming
'''


class MoosePoetryRankingSpider(scrapy.Spider):
    name = 'MoosePoetryRankingSpider'

    allowed_domains = ['www.shicimingju.com']

    start_urls = ['https://www.shicimingju.com/paiming']

    def parse(self, response):
        ranking_item = MoosePoetryRankingItem()

        poetry_list = response.xpath('//div[@id="main_left"]/div[@class="card shici_card"]').extract()
        if len(poetry_list) > 0:
            for poetry_category in poetry_list:
                ranking_num = Selector(text=poetry_category).xpath(
                    '//div[@class="list_num_info"]/span/text()').extract_first()
                ranking_item['ranking_num'] = int(ranking_num)

                poetry_dynasty_list = Selector(text=poetry_category).xpath('//div[@class="list_num_info"]/text()').extract()
                if len(poetry_dynasty_list) == 3:
                    dynasty = poetry_dynasty_list[1]
                    ranking_item['dynasty'] = dynasty

                author_name = Selector(text=poetry_category).xpath(
                    '//div[@class="list_num_info"]/a/text()').extract_first()
                ranking_item['author_name'] = author_name

                poetry_name = Selector(text=poetry_category).xpath(
                    '//div[@class="shici_list_main"]/h3/a/text()').extract_first()
                ranking_item['poetry_name'] = poetry_name
                html = Selector(text=poetry_category).xpath(
                    '//div[@class="shici_list_main"]/div[@class="shici_content"]').extract_first()
                if html is not None:
                    pattern = re.compile(r'<[^>]+>', re.S)
                    poetry_content = pattern.sub("", html).replace("展开全文", "").replace("收起", "").replace(" ", "")
                    # .replace("\r", "")
                    ranking_item['poetry_content'] = poetry_content
                yield ranking_item

        # 获取下一页数据
        next_url = response.xpath(
            "//div[@id='main_left']/div[@id='list_nav']/div[@id='list_nav_part']/a[text()='下一页']/@href").extract_first()

        # 抓取下一页数据
        if next_url is not None:
            yield scrapy.Request(str.format("https://www.shicimingju.com{}", next_url), callback=self.parse,
                                 dont_filter=True)
